# as soon as possible (usually as soon as the tag is closed) to reduce
# its memory footprint.
+# We've been gradually lifting some of these restrictions to handle
+# non-sanitized output generated by extensions, but we shortcut the tokenizer
+# for speed (primarily by splitting on `<`) and so rely on syntactic
+# well-formedness.
+
# On the other hand, I've been pretty careful to note with comments in the
# code the places where this implementation omits features of the spec or
# depends on the MediaWiki Sanitizer. Perhaps in the future we'll want to
self::HTML_NAMESPACE => [
'html' => true, 'head' => true, 'body' => true, 'frameset' => true,
'frame' => true,
- 'plaintext' => true, 'isindex' => true, 'textarea' => true,
+ 'plaintext' => true, 'isindex' => true,
'xmp' => true, 'iframe' => true, 'noembed' => true,
'noscript' => true, 'script' => true,
'title' => true
]
];
+ public static $extraLinefeedSet = [
+ self::HTML_NAMESPACE => [
+ 'pre' => true, 'textarea' => true, 'listing' => true,
+ ]
+ ];
+
public static $headingSet = [
self::HTML_NAMESPACE => [
'h1' => true, 'h2' => true, 'h3' => true,
}
if ( !$this->isA( BalanceSets::$emptyElementSet ) ) {
$out = "<{$this->localName}{$encAttribs}>";
+ $len = strlen( $out );
// flatten children
foreach ( $this->children as $elt ) {
$out .= "{$elt}";
}
$out .= "</{$this->localName}>";
+ if (
+ $this->isA( BalanceSets::$extraLinefeedSet ) &&
+ $out[$len] === "\n"
+ ) {
+ // Double the linefeed after pre/listing/textarea
+ // according to the HTML5 fragment serialization algorithm.
+ $out = substr( $out, 0, $len + 1 ) .
+ substr( $out, $len );
+ }
} else {
$out = "<{$this->localName}{$encAttribs} />";
Assert::invariant(
return $out;
}
+ /**
+ * Insert a comment at the appropriate place for inserting a node.
+ * @param string $value Content of the comment.
+ * @see https://html.spec.whatwg.org/multipage/syntax.html#insert-a-comment
+ */
+ public function insertComment( $value ) {
+ // Just another type of text node, except for tidy p-wrapping.
+ return $this->insertText( '<!--' . $value . '-->', true );
+ }
+
/**
* Insert text at the appropriate place for inserting a node.
* @param string $value
* @see https://html.spec.whatwg.org/multipage/syntax.html#appropriate-place-for-inserting-a-node
*/
- public function insertText( $value ) {
+ public function insertText( $value, $isComment = false ) {
if (
$this->fosterParentMode &&
$this->currentNode->isA( BalanceSets::$tableSectionRowSet )
) {
$this->fosterParent( $value );
} elseif (
- $this->tidyCompat &&
+ $this->tidyCompat && !$isComment &&
$this->currentNode->isA( BalanceSets::$tidyPWrapSet )
) {
$this->insertHTMLELement( 'mw:p-wrap', [] );
* - The document is never in "quirks mode".
* - All occurrences of < and > have been entity escaped, so we
* can parse tags by simply splitting on those two characters.
+ * (This also simplifies the handling of < inside <textarea>.)
+ * The character < must not appear inside comments.
* Similarly, all attributes have been "cleaned" and are double-quoted
* and escaped.
- * - All comments and null characters are assumed to have been removed.
- * - We don't alter linefeeds after <pre>/<listing>.
+ * - All null characters are assumed to have been removed.
* - The following elements are disallowed: <html>, <head>, <body>, <frameset>,
- * <frame>, <plaintext>, <isindex>, <textarea>, <xmp>, <iframe>,
+ * <frame>, <plaintext>, <isindex>, <xmp>, <iframe>,
* <noembed>, <noscript>, <script>, <title>. As a result,
* further simplifications can be made:
* - `frameset-ok` is not tracked.
* - `head element pointer` is not tracked (but presumed non-null)
- * - Tokenizer has only a single mode.
+ * - Tokenizer has only a single mode. (<textarea> wants RCDATA and
+ * <style>/<noframes> want RAWTEXT modes which we only loosely emulate.)
*
* We generally mark places where we omit cases from the spec due to
* disallowed elements with a comment: `# OMITTED: <element-name>`.
private $stack;
private $strict;
private $tidyCompat;
+ private $allowComments;
- private $textIntegrationMode = false;
+ private $textIntegrationMode;
private $pendingTableText;
private $originalInsertionMode;
private $fragmentContext;
private $formElementPointer;
+ private $ignoreLinefeed;
+ private $inRCDATA;
+ private $inRAWTEXT;
+
+ /**
+ * Valid HTML5 comments.
+ * Regex borrowed from Tim Starling's "remex-html" project.
+ */
+ const VALID_COMMENT_REGEX = "~ !--
+ ( # 1. Comment match detector
+ > | -> | # Invalid short close
+ ( # 2. Comment contents
+ (?:
+ (?! --> )
+ (?! --!> )
+ (?! --! \z )
+ (?! -- \z )
+ (?! - \z )
+ .
+ )*+
+ )
+ ( # 3. Comment close
+ --> | # Normal close
+ --!> | # Comment end bang
+ ( # 4. Indicate matches requiring EOF
+ --! | # EOF in comment end bang state
+ -- | # EOF in comment end state
+ - | # EOF in comment end dash state
+ # EOF in comment state
+ )
+ )
+ )
+ ([^<]*) \z # 5. Non-tag text after the comment
+ ~xs";
/**
* Create a new Balancer.
* program: <p>-wrapping is done to the children of
* <body> and <blockquote> elements, and empty elements
* are removed.
+ * 'allowComments': boolean, defaults to true.
+ * When true, allows HTML comments in the input.
+ * The Sanitizer generally strips all comments, so if you
+ * are running on sanitized output you can set this to
+ * false to get a bit more performance.
*/
public function __construct( array $config = [] ) {
$config = $config + [
'strict' => false,
'allowedHtmlElements' => null,
'tidyCompat' => false,
+ 'allowComments' => true,
];
$this->allowedHtmlElements = $config['allowedHtmlElements'];
$this->strict = $config['strict'];
$this->tidyCompat = $config['tidyCompat'];
+ $this->allowComments = $config['allowComments'];
if ( $this->allowedHtmlElements !== null ) {
# Sanity check!
$bad = array_uintersect_assoc(
$this->processingCallback = $processingCallback;
$this->processingArgs = $processingArgs;
+ $this->textIntegrationMode =
+ $this->ignoreLinefeed =
+ $this->inRCDATA =
+ $this->inRAWTEXT = false;
+
# The stack is constructed with an <html> element already on it.
# Set this up as a fragment parsed with <body> as the context.
$this->fragmentContext =
# Don't actually inject the empty string as a text token.
return true;
}
+ // Support pre/listing/textarea by suppressing initial linefeed
+ if ( $this->ignoreLinefeed ) {
+ $this->ignoreLinefeed = false;
+ if ( $token === 'text' ) {
+ if ( $value[0] === "\n" ) {
+ if ( $value === "\n" ) {
+ # Nothing would be left, don't inject the empty string.
+ return true;
+ }
+ $value = substr( $value, 1 );
+ }
+ }
+ }
// Some hoops we have to jump through
$adjusted = $this->stack->adjustedCurrentNode( $this->fragmentContext );
/**
* Grab the next "token" from $bitsIterator. This is either a open/close
- * tag or text, depending on whether the Sanitizer approves.
+ * tag or text or a comment, depending on whether the Sanitizer approves.
*/
private function advance() {
$x = $this->bitsIterator->current();
$this->bitsIterator->next();
$regs = [];
+ # Handle comments. These won't be generated by mediawiki (they
+ # are stripped in the Sanitizer) but may be generated by extensions.
+ if (
+ $this->allowComments &&
+ !( $this->inRCDATA || $this->inRAWTEXT ) &&
+ preg_match( Balancer::VALID_COMMENT_REGEX, $x, $regs, PREG_OFFSET_CAPTURE ) &&
+ /* verify EOF condition where necessary */
+ ( $regs[4][1] < 0 || !$this->bitsIterator->valid() )
+ ) {
+ $contents = $regs[2][0];
+ $rest = $regs[5][0];
+ $this->insertToken( 'comment', $contents );
+ $this->insertToken( 'text', str_replace( '>', '>', $rest ) );
+ return;
+ }
# $slash: Does the current element start with a '/'?
# $t: Current element name
# $attribStr: String between element name and >
$slash = $t = $attribStr = $brace = $rest = null;
}
$goodtag = $t;
+ if ( $this->inRCDATA ) {
+ if ( $slash && $t === $this->inRCDATA ) {
+ $this->inRCDATA = false;
+ } else {
+ // No tags allowed; this emulates the "rcdata" tokenizer mode.
+ $goodtag = false;
+ }
+ }
+ if ( $this->inRAWTEXT ) {
+ if ( $slash && $t === $this->inRAWTEXT ) {
+ $this->inRAWTEXT = false;
+ } else {
+ // No tags allowed, no entity-escaping done.
+ $goodtag = false;
+ }
+ }
$sanitize = $this->allowedHtmlElements !== null;
if ( $sanitize ) {
$goodtag = $t && isset( $this->allowedHtmlElements[$t] );
if ( $goodtag ) {
$rest = str_replace( '>', '>', $rest );
$this->insertToken( 'text', str_replace( '>', '>', $rest ) );
+ } elseif ( $this->inRAWTEXT ) {
+ $this->insertToken( 'text', "<$x" );
} else {
# bad tag; serialize entire thing as text.
$this->insertToken( 'text', '<' . str_replace( '>', '>', $x ) );
private function parseRawText( $value, $attribs = null ) {
$this->stack->insertHTMLElement( $value, $attribs );
- // XXX switch tokenizer to rawtext state?
+ $this->inRAWTEXT = $value;
$this->originalInsertionMode = $this->switchMode( 'inTextMode' );
return true;
}
// ignore any other end tag
return true;
}
+ } elseif ( $token === 'comment' ) {
+ $this->stack->insertComment( $value );
+ return true;
}
// If not handled above
$this->inBodyMode( 'endtag', 'p' );
}
$this->stack->insertHTMLElement( $value, $attribs );
- # As described in "simplifications" above:
- # 1. We don't touch the next token, even if it's a linefeed.
- # 2. OMITTED: frameset_ok
+ $this->ignoreLinefeed = true;
+ # OMITTED: frameset_ok
return true;
case 'form':
return $this->inBodyMode( $token, 'img', $attribs, $selfclose );
# OMITTED: <isindex>
- # OMITTED: <textarea>
+
+ case 'textarea':
+ $this->stack->insertHTMLElement( $value, $attribs );
+ $this->ignoreLinefeed = true;
+ $this->inRCDATA = $value; // emulate rcdata tokenizer mode
+ # OMITTED: frameset_ok
+ return true;
+
# OMITTED: <xmp>
# OMITTED: <iframe>
# OMITTED: <noembed>
}
}
return true;
+ } elseif ( $token === 'comment' ) {
+ $this->stack->insertComment( $value );
+ return true;
} else {
Assert::invariant( false, "Bad token type: $token" );
}
return $this->inHeadMode( $token, $value, $attribs, $selfclose );
}
// Fall through for "anything else" clause.
+ } elseif ( $token === 'comment' ) {
+ $this->stack->insertComment( $value );
+ return true;
}
// This is the "anything else" case:
$this->stack->fosterParentMode = true;
// Fall through for "anything else".
} elseif ( $token === 'eof' ) {
return $this->inBodyMode( $token, $value, $attribs, $selfclose );
+ } elseif ( $token === 'comment' ) {
+ $this->stack->insertComment( $value );
+ return true;
}
// Anything else
case 'template':
return $this->inHeadMode( $token, $value, $attribs, $selfclose );
}
+ } elseif ( $token === 'comment' ) {
+ $this->stack->insertComment( $value );
+ return true;
}
// anything else: just ignore the token
return true;
}
private function inTemplateMode( $token, $value, $attribs = null, $selfclose = false ) {
- if ( $token === 'text' ) {
+ if ( $token === 'text' || $token === 'comment' ) {
return $this->inBodyMode( $token, $value, $attribs, $selfclose );
} elseif ( $token === 'eof' ) {
if ( $this->stack->indexOf( 'template' ) < 0 ) {
'strict' => false, /* not strict */
'allowedHtmlElements' => null, /* no sanitization */
'tidyCompat' => false, /* standard parser */
+ 'allowComments' => true, /* comment parsing */
] );
}
/**
- * Anything cleanup you need to do should go here.
- */
- protected function tearDown() {
- parent::tearDown();
- }
-
- /**
- * @covers Balancer::balance
+ * @covers MediaWiki\Tidy\Balancer::balance
* @dataProvider provideBalancerTests
*/
public function testBalancer( $description, $input, $expected ) {
// for providers, and filter out HTML constructs which
// the balancer doesn't support.
$tests = [];
- $start = '<html><head></head><body>';
- $end = '</body></html>';
+ $okre = "~ \A
+ (?i:<!DOCTYPE\ html>)?
+ <html><head></head><body>
+ .*
+ </body></html>
+ \z ~xs";
foreach ( $json as $filename => $cases ) {
foreach ( $cases as $case ) {
$html = $case['document']['html'];
- if (
- substr( $html, 0, strlen( $start ) ) !== $start ||
- substr( $html, -strlen( $end ) ) !== $end
- ) {
+ if ( !preg_match( $okre, $html ) ) {
// Skip tests which involve stuff in the <head> or
// weird doctypes.
continue;
$html = $case['document']['noQuirksBodyHtml'];
// Normalize case of SVG attributes.
$html = str_replace( 'foreignObject', 'foreignobject', $html );
+ // Normalize case of MathML attributes.
+ $html = str_replace( 'definitionURL', 'definitionurl', $html );
- if ( isset( $case['document']['props']['comment'] ) ) {
- // Skip tests which include HTML comments, which
- // the balancer requires to have been stripped.
+ if (
+ isset( $case['document']['props']['comment'] ) &&
+ preg_match( ',<!--[^>]*<,', $html )
+ ) {
+ // Skip tests which include HTML comments containing
+ // the < character, which we don't support.
continue;
}
if ( strpos( $case['data'], '<![CDATA[' ) !== false ) {
// Skip tests involving <![CDATA[ ]]> quoting.
continue;
}
- if ( stripos( $case['data'], '<!DOCTYPE' ) !== false ) {
- // Skip tests involving doctypes.
+ if (
+ stripos( $case['data'], '<!DOCTYPE' ) !== false &&
+ stripos( $case['data'], '<!DOCTYPE html>' ) === false
+ ) {
+ // Skip tests involving unusual doctypes.
continue;
}
- if ( preg_match( ',</?(html|head|body|frame|plaintext)>|<rdar:|<isindex,i', $case['data'] ) ) {
+ $literalre = "~ <rdar: | <isindex | < /? (
+ html | head | body | frame | frameset | plaintext
+ ) > ~xi";
+ if ( preg_match( $literalre, $case['data'] ) ) {
// Skip tests involving some literal tags, which are
// unsupported but don't show up in the expected output.
continue;
isset( $case['document']['props']['tags']['script'] ) ||
isset( $case['document']['props']['tags']['svg script'] ) ||
isset( $case['document']['props']['tags']['svg title'] ) ||
- isset( $case['document']['props']['tags']['textarea'] ) ||
isset( $case['document']['props']['tags']['title'] ) ||
isset( $case['document']['props']['tags']['xmp'] )
) {
isset( $case['document']['props']['tagWithLt'] ) ||
isset( $case['document']['props']['attrWithFunnyChar'] ) ||
preg_match( ':^(</b test|<di|<foo bar=qux/>)$:', $case['data'] ) ||
- preg_match( ':</p<p>:', $case['data'] )
+ preg_match( ':</p<p>:', $case['data'] ) ||
+ preg_match( ':<b &=&>|<p/x/y/z>:', $case['data'] )
) {
// Skip tests with funny tag or attribute names,
// which are really tests of the HTML tokenizer, not
continue;
}
if (
- stripos( $case['data'], 'encoding=" text/html "' ) !== false
+ preg_match( ':encoding=" text/html "|type=" hidden":', $case['data'] )
) {
// The Sanitizer normalizes whitespace in attribute
// values, which makes this test case invalid.
// Skip tests with ASCII null, etc.
continue;
}
+ $data = preg_replace(
+ '~<!DOCTYPE html>~i', '', $case['data']
+ );
$tests[] = [
$filename, # use better description?
- $case['data'],
+ $data,
$html
];
}
dépôts / lhc / web / wiklou.git / commitdiff